#coding:utf-8
import requests, logging, datetime
from ipyparallel import Client
from pymongo.errors import DuplicateKeyError
# client = Client('security/ipcontroller-client.json')
# client = Client()
hash_dict = {}
from zhihu_lxml import User as Zhihu_User
from time import sleep
from model.user import User, SearchFolloweeUser, SearchFollowerUser
from zhihu_lxml import retry_request, requests, parser_user, parser_answer
def datetime2str(datetime):
    return datetime.strftime('%Y-%m-%d')
get_user_data = list(User.col.find(
        {'$or': [{'date': {'$exists': False}}, {'date': datetime2str(datetime.date.today())}]}
    ).distinct('url'))
hash_dict = {i: 0 for i in get_user_data if 'https' in i}
print len(hash_dict.keys()), "exists", hash_dict.keys()[0]


def task_job(content):
    return content[:10]

def remote_task_user(url):
    r = retry_request(requests.get,url + '/about')
    if r is None:
        return False, url
    return parser_user(url, r.content)
    # except Exception,e:
    #     print "erro request"
    #     return False, url
# print remote_task('https://www.zhihu.com/people/amuro1230')

# client[:].push(dict(parser_user=parser_user))
# r = client[:].map_sync(remote_task, ['http://www.baidu.com','http://hao123.com']*20)
start_user_url = [u'https://www.zhihu.com/people/jixin',
 u'https://www.zhihu.com/people/zhouyuan',
 u'https://www.zhihu.com/people/yolfilm',
 u'https://www.zhihu.com/people/zhu-xuan-86',
 u'https://www.zhihu.com/people/zhang-jia-wei',
 u'https://www.zhihu.com/people/magie',
 u'https://www.zhihu.com/people/imike',
 u'https://www.zhihu.com/people/kaifulee',
 u'https://www.zhihu.com/people/gejinyuban',
 u'https://www.zhihu.com/people/amuro1230',
 u'https://www.zhihu.com/people/zhang-xiao-bei',
 u'https://www.zhihu.com/people/cai-tong']
loop_index = 0
if __name__ == "__main__":
    for i in start_user_url:
        print "parser", i
        user_same_num = 0
        u = Zhihu_User(i)
        followers = u.get_followers_url()
        for index, url in enumerate(followers):
            if index % 19 ==0:
                if index != 0:
                    results = map(remote_task_user, temp)
                    for result, data in results:
                        if result is True:
                            try:
                                data['date'] = datetime2str(datetime.date.today())
                                User.col.insert(data)
                                hash_dict[data['url']] = 1
                                print "insert", data['url']
                            except DuplicateKeyError,e:
                                print e
                                continue

                temp = []
            if 'https' not in url:
                url = url.replace('http', 'https')
            if url in hash_dict:
                user_same_num += 0
                continue
            if user_same_num > 200:
                break
            temp.append(url)


    # statr_user_url = User.col.find({'date': {'$exists': False},'followers_num': {'$gte': 500000}}).distinct('url')
    # for url in get_user_data:
    #     start_user  = Zhihu_User(url)
    #     print url
    #     if SearchFolloweeUser.col.find_one({'url': url}):
    #         print "pass"
    #         continue
    #     for i in start_user.get_followees():
    #         try:
    #             # if loop_index % 200 == 0:
    #                 # url_list = [url for url in url_list if not is_in_hash_dict(url)]
    #                 # print url_list, len(url_list)
    #             if not is_in_hash_dict(i.user_url):
    #                 result, url = remote_task(i.user_url)
    #                 if result == True:
    #                     hash_dict[i.user_url] = 0
    #                 else:
    #                     hash_dict.pop(i.user_url)
    #         except Exception,e:
    #             continue
    #     SearchFolloweeUser.col.insert({
    #         'url': url,
    #         'count': 1
    #     })
